%% @author Rob Evans <rob.g.evans@gmail.com>
%% @copyright 2008 ftw

%% @doc Using Mochiweb's HTML parser, pulls out XFN links and updates CouchDB
-module(es_parse).
-export([start/2,
		 test/0,
		 receiver/0,
		 splitUrl/1]).

%% Testing/debugging macros
-define(TEST(Call), io:format("~s: ~w~n", [??Call, Call])).
-define(DEBUGMODE, false).
-define(DEBUG(Stmt), if ?DEBUGMODE==true -> io:format("~s~n",[Stmt]); true -> ok end).

%% Helper function - same as lists:keytake
%% Provided for compatibility with older versions of erlang

keytake(_, _, []) ->
    false;

keytake(Key, N, [Tuple|Rest]) ->
    case element(N, Tuple) of
        Key ->
            {value, Tuple, Rest};
        _ ->
            keytake(Key, N, Rest)
    end.

% IPC testing
receiver() ->
	receive 
		quit ->
			ok;
		Res ->
			if
				is_atom(Res) ->
					Disp=atom_to_list(Res);
				is_list(Res) ->
					Disp=Res;
				is_pid(Res) ->
					Disp=pid_to_list(Res);
				true ->
					Disp="unknown type"
			end,
			io:format(pid_to_list(self())++"\t\"~s\" received\n", [Disp]),
			receiver()
	end.

%% @spec start(pid(),string()) -> void
%% @doc takes in its parents pid and a URL to crawl, checks robots.txt, crawls URL and updates database
start(Parent,Url) ->
	
%	Robots = (catch es_robots:checkrobots_txt(Url)),

%	case Robots of
%		{'EXIT', _Term} ->
%			?DEBUG("{error, checkrobots}"),
%			Parent ! completed;
%		false ->
%			?DEBUG("{robots, disallowed}"),
%			Parent ! completed;
%		true ->
%			?DEBUG("{robots, allowed}"),
			Res = (catch fetch_and_update(Url)),
			case Res of
				{'EXIT', _Term} ->
					?DEBUG("{error, fetch_and_update}"),
					Parent ! completed;
		%			Parent ! {error, Res};
				_ ->
					?DEBUG("{ok, es_parse:start}"),
					Parent ! completed
		%			Parent ! {success, Url}
			end.
%	end.

%% @spec fetch_and_update(string()) -> void
%% @doc takes URL, fetches page, finds XFN links and updates database
fetch_and_update(Url) ->
	inets:start(),
	{_Type,CurServer,CurPath,_File} = splitUrl(Url),
	put(url,Url),
	put(server,CurServer),
	put(path,CurPath),
	{ok,{_,_,Body}}=http:request(Url),
	Tree=mochiweb_html:parse(Body),
	ThreeTuples = getLinks(Tree),
    
	case ?DEBUGMODE of
		false ->
			es_db:add_links(Url, ThreeTuples),
			ok;
		true ->
			ok
	end.

%% @spec getLinks({ string() | binary(), [ { binary(), binary() } ], [ html_node() | string() | binary() ] }) -> [ { string(), string(), string() } ]
%% @doc given a document tree from mochiweb_html:parse it finds and returns the XFN links
getLinks({Tag,Properties,Contents}) ->
	GetLinks=fun({SmTag,SmProperties,SmContents}) ->
		getLinks({SmTag,SmProperties,SmContents}); (_) -> [] end,
	
	CheckedTag = string:to_lower(erlang:binary_to_list(Tag)),
				
	SubLinks = compareAndDrop(lists:merge(lists:map(GetLinks,Contents))),

	if
		CheckedTag == "a" ->
			Links = processProperties(Properties),
			if
				is_list(Links) ->
					compareAndDrop(Links++SubLinks);
				Links == false ->
					SubLinks
			end;
		true ->
			SubLinks
	end;

getLinks(_) ->
	[].

%% @spec processProperties([ { binary() | binary() } ]) -> [{ string(), string(), string() }]
%% @doc returns all database tuples for a given link (possibily multiple tuples, one for each relation)
processProperties(ListOfProperties) ->
		case keytake(<<"href">>,1,ListOfProperties) of
			{value,{_,HREF},_} ->
				FULLHREF=partialToFullUrl(erlang:binary_to_list(HREF));
			_ ->
				FULLHREF=false,
				HREF=false
		end,
	
		case keytake(<<"rel">>,1,ListOfProperties) of
			{value,{_,REL},_} ->
				REL_LIST=erlang:binary_to_list(REL),
				GOODXFN=goodXFN(REL_LIST);	
			_ ->
				REL_LIST=false,
				REL=false,
				GOODXFN=false
		end,

		if
			(HREF==false) orelse (REL==false) orelse (GOODXFN==false) ->
				false;
			true ->
				URL=get(url),
				Dup=fun(Relation) -> {URL, FULLHREF, Relation} end,
				lists:map(Dup,string:tokens(REL_LIST," "))
		end.

%% @spec partialToFullUrl(string()) -> string()
%% @doc given a partially specified URL, return a full, absolute URL given the current server and path
partialToFullUrl(Link) ->
	CurrentServer = get(server),
	CurrentDir = get(path),
	case Link of
		"http://"++_ ->
			Link;
		"//"++_ ->
			"http:"++Link;
		"/"++_ ->
			"http://"++CurrentServer++Link;
		_ ->
			"http://"++CurrentServer++CurrentDir++Link
	end.

%% @spec splitUrl(string()) -> { atom(), string(), string(), string() }
%% @doc given a full, absolute URL, returns a decomposition into the server, path and filename
splitUrl(FullUrl) ->
	"http://"++Url=FullUrl,
	SplitAt=string:chr(Url,$/),
	if
		SplitAt > 0 ->
			{Server,Filepath}={string:substr(Url,1,SplitAt-1),string:substr(Url,SplitAt)},
			SplitAt2=string:rchr(Filepath,$/);
		SplitAt == 0 ->
			Server = Url,
			Filepath = [],
			SplitAt2 = 0
	end,
			
	if
		SplitAt2 > 0 ->
			{Path,File}={string:substr(Filepath,1,SplitAt2),string:substr(Filepath,SplitAt2+1)};
		SplitAt2 == 0 ->
			Path = "/",
			File = []
	end,
	case length(File) of
		0 ->
			Type=path;
		_ ->
			Type=file
	end,
	{Type,Server,Path,File}.

%% @spec compareAndDrop([item()]) -> [item()]
%% @doc given a list of items, compares and drops duplicate items which are next to each other
compareAndDrop(List) ->
	case List of
		[] ->
			[];

		[ _First ] ->
			List;
			
		[ First, Second ] ->
			if
				First == Second ->
					[ First ];
				true ->
					List
			end;
			
		[ First, Second | Rest ] ->
			if
				First == Second ->
					compareAndDrop([First | Rest ]);
				true ->
					[ First | compareAndDrop([Second | Rest ])]
			end

	end.

%% @spec goodXFN(string()) -> bool()
%% @doc given a string containing one or more space-separated XFN relations, returns true if all are valid, false otherwise
goodXFN(XFNRel) ->
	IsXFNWord=fun(XFNWord) ->
		lists:member(XFNWord,["contact",
							  "acquaintance",
							  "friend",
							  "met",
							  "co-worker",
							  "colleague",
							  "co-resident",
							  "neighbor",
							  "child",
							  "parent",
							  "sibling",
							  "spouse",
							  "kin",
							  "muse",
							  "crush",
							  "date",
							  "sweetheart",
							  "me"])
		end,

	lists:all(IsXFNWord,string:tokens(XFNRel," ")).

	
%%%%%%%%%%% Testing functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

test_getLinks() ->
	Tree = { <<"html">>, [],
				[ {<<"head">>, [], []},
				  {<<"body">>, [],
				  				[ {<<"h1">>,[],[<<"this">>, { <<"a">>, [ {<<"href">>,<<"http://link1.com/">>},{<<"rel">>,<<"me">>}], [<<"hiya">>] }, <<"cool">>]},
				  				  {<<"a">>,[{<<"href">>,<<"http://link2.com/">>}],[<<"no rel">>]}
				  				]}]
			},
	[ {"http://link1.com/","me"} ] = getLinks(Tree),
	ok.

test_processProperties() ->
	A = { <<"href">>, <<"//google.com/">> },
	B = { <<"rel">>, <<"me">> },
	C = { <<"href">>, <<"http://eflorenzano.com/">> },
	D = { <<"rel">>, <<"friend">> },
	false = processProperties([A]),
	false = processProperties([B]),
	{"http://google.com/","me"} = processProperties([A,B]),
	{"http://eflorenzano.com/","friend"} = processProperties([C,D]),
	ok.

test_partialToFullUrl() ->
	OldServer = put(server, "eflorenzano.com"),
	OldDir = put(path, "/subdir/"),
	
	"http://www.snapple.com/file.html" = partialToFullUrl("//www.snapple.com/file.html"),
	"http://google.com/file.html" = partialToFullUrl("//google.com/file.html"),
	"http://eflorenzano.com/newdir/file.html" = partialToFullUrl("/newdir/file.html"),
	"http://eflorenzano.com/subdir/file.html" = partialToFullUrl("file.html"),
	
	put(server,OldServer),
	put(path,OldDir),
	ok.

test_splitUrl() ->
	{path, "eflorenzano.com", "/", []} = splitUrl("http://eflorenzano.com"),
	{path, "eflorenzano.com", "/", []} = splitUrl("http://eflorenzano.com/"),
	{file, "eflorenzano.com", "/", "subdir" } = splitUrl("http://eflorenzano.com/subdir"),
	{path, "eflorenzano.com", "/subdir/", []} = splitUrl("http://eflorenzano.com/subdir/"),
	{file, "eflorenzano.com", "/subdir/", "file.html"} = splitUrl("http://eflorenzano.com/subdir/file.html"),
	ok.

test_compareAndDrop() ->
	List = [1,3,4,5,6],
	[] = compareAndDrop([]),
	[2] = compareAndDrop([2,2]),
	List = compareAndDrop(List),
	List = compareAndDrop([1|List]),
	ok.
	
test_goodXFN() ->
	true = goodXFN("kin"),
	true = goodXFN("contact met"),
	false = goodXFN("elephant"),
	false = goodXFN("me sexy"),
	ok.

test() ->
	?TEST(test_getLinks()),
	?TEST(test_processProperties()),
	?TEST(test_partialToFullUrl()),
	?TEST(test_splitUrl()),
	?TEST(test_compareAndDrop()),
	?TEST(test_goodXFN()),
	ok.